﻿module Links

open System
open System.Text.RegularExpressions
  
/// Extracts links from HTML (canonicalization is forced).
let extractAllLinks (uri : Uri) html =

    let uri' = string uri
    let host = uri.Host
    let domPat = "[^\.]+\.\w{2,3}(\.\w{2})?"
    let domain = Regex(domPat, RegexOptions.RightToLeft).Match(host).Value
 
    let pattern1 = "(?i)href\\s*=\\s*(\"|\')/?((?!#.*|/\B|mailto:|location\.|javascript:)[^\"\'\#]+)(\"|\'|\#)"   
    let pattern2 = "(?i)^https?://[^\"]*"
    let pattern3 = "(?i)https?://" + Regex.Escape(domain) + "[^\"]*"
    let pattern4 = "(?i)^https?://((www\.)|([^\.]+\.))?" + Regex.Escape(domain) + "[^\"]*"
 
    let absolute, relative =
        [
            for x in Regex(pattern1).Matches(html) do
                yield x.Groups.[2].Value
        ]
        |> List.partition (fun x -> Regex(pattern2).IsMatch(x))
 
    let relative' = relative |> List.map (fun x -> "http://" + host + "/" + x)
    let absolute' = absolute |> List.map (fun x -> Regex("[^\#]+").Match(x).Value)
 
    let allLinks = absolute' @ relative' |> List.map (fun x -> let m = Regex(pattern3).IsMatch(x)
                                                               match m with
                                                               | true -> Regex("(?i)https?://").Replace(x, "http://www.")
                                                               | false -> x)
                                         |> List.map (fun x -> new Uri(x))
                                         |> List.map (fun x -> x.ToString())
                                         |> Seq.distinct
                                         |> Seq.toList
    let intLinks, extLinks = allLinks |> List.partition (fun x -> Regex(pattern4).IsMatch(x))
    intLinks, extLinks